{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://githubtocolab.com/CRCTransformers/deepdive-book/blob/main/Chapter-3-RoBERTa_SentimentClassification.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Utk9qp_jjowv"
   },
   "source": [
    "# Setup\n",
    "\n",
    "This notebook inspired by/modified from: Rukshar Alam (https://github.com/rukshar69/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Bx1rOj7yBKlN",
    "outputId": "3d1bb2e6-17cc-4d94-c9e8-be8c30d790c8"
   },
   "outputs": [],
   "source": [
    "!pip install transformers==4.16.2 tokenizers==0.10.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pM5JkntzA8TW",
    "outputId": "281eedcf-f45a-440f-f026-1b4314917f8e"
   },
   "outputs": [],
   "source": [
    "from transformers import RobertaTokenizer, \\\n",
    "                         RobertaModel, \\\n",
    "                         get_linear_schedule_with_warmup\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "from torch import nn, optim\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "\n",
    "from collections import defaultdict\n",
    "from textwrap import wrap\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "sns.set_palette(sns.color_palette(\"Paired\"))\n",
    "\n",
    "rcParams['figure.figsize'] = 12,8\n",
    "RANDOM_SEED = 42\n",
    "np.random.seed(RANDOM_SEED)\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VdsMuXmuj2Eh"
   },
   "source": [
    "# Read Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 581
    },
    "id": "GkdM3VKzGkUZ",
    "outputId": "a016f0c1-3687-4310-8d26-4d46b80bf4e6"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"reviews.csv\")\n",
    "\n",
    "# Map sentiment scores to categories\n",
    "def map_sentiment_scores(score_value):\n",
    "    score_value = int(score_value)\n",
    "    if score_value<=2:\n",
    "        return 0\n",
    "    elif score_value == 3:\n",
    "        return 1\n",
    "    else:\n",
    "        return 2\n",
    "    \n",
    "    \n",
    "df['sentiment'] = df.score.apply(map_sentiment_scores)\n",
    "class_names = ['negative', 'neutral', 'positive']\n",
    "\n",
    "ax = sns.countplot(df.sentiment)\n",
    "plt.xlabel('review sentiment')\n",
    "ax.set_xticklabels(class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qxt7wDO7j40E"
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dX_g6AEzj6pR"
   },
   "source": [
    "## Define Transformer and Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K9UzAgFKOEQN"
   },
   "outputs": [],
   "source": [
    "PRE_TRAINED_MODEL_NAME = 'roberta-base'\n",
    "tokenizer = RobertaTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-896Jhauj-Ma"
   },
   "source": [
    "### Tokenization and encoding example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "G5zNdnhpPPFG",
    "outputId": "8df10877-5da5-4613-8dd0-d5965e53ab5a"
   },
   "outputs": [],
   "source": [
    "sample_txt = 'When was I last outside? I am stuck at home for 2 weeks.'\n",
    "tokens = tokenizer.tokenize(sample_txt)\n",
    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(f' Sentence: {sample_txt}')\n",
    "print(f'   Tokens: {tokens}')\n",
    "print(f'Token IDs: {token_ids}')\n",
    "\n",
    "encoding = tokenizer.encode_plus(\n",
    "    sample_txt,\n",
    "    max_length=32,\n",
    "    truncation=True,\n",
    "    add_special_tokens=True, # Add '[CLS]' and '[SEP]'\n",
    "    return_token_type_ids=False,\n",
    "    padding=True,\n",
    "    return_attention_mask=True,\n",
    "    return_tensors='pt')  # Return PyTorch tensors)\n",
    "\n",
    "print(f'Encoding keys: {encoding.keys()}')\n",
    "print(len(encoding['input_ids'][0]))\n",
    "print(encoding['input_ids'][0])\n",
    "print(len(encoding['attention_mask'][0]))\n",
    "print(encoding['attention_mask'])\n",
    "print(tokenizer.convert_ids_to_tokens(encoding['input_ids'][0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kGCthVhzkXov"
   },
   "source": [
    "### EDA of token counts in the reviews dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 581
    },
    "id": "2N872LZ9Rn2l",
    "outputId": "8fbaa685-715b-49b2-d834-210844a05fc2"
   },
   "outputs": [],
   "source": [
    "token_lens = []\n",
    "for txt in df.content:\n",
    "    tokens = tokenizer.encode(txt, truncation=True, max_length=512)\n",
    "    token_lens.append(len(tokens))\n",
    "\n",
    "sns.distplot(token_lens)\n",
    "plt.xlim([0,256])\n",
    "plt.xlabel('Token count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "podvfmBOkdgw"
   },
   "source": [
    "# Dataset Utility Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     3
    ],
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Tb8ttnGmTViV",
    "outputId": "f12ed147-1f08-4989-9654-295be0428f53"
   },
   "outputs": [],
   "source": [
    "MAX_LEN = 160\n",
    "BATCH_SIZE = 16\n",
    "\n",
    "\n",
    "class GPReviewDataset(Dataset):\n",
    "    def __init__(self, reviews, targets, tokenizer, max_len, include_raw_text=False):\n",
    "        self.reviews = reviews\n",
    "        self.targets = targets\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_len = max_len\n",
    "        self.include_raw_text = include_raw_text\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.reviews)\n",
    "    \n",
    "    def __getitem__(self, item):\n",
    "        review = str(self.reviews[item])\n",
    "        target = self.targets[item]\n",
    "        encoding = self.tokenizer.encode_plus(\n",
    "            review, \n",
    "            add_special_tokens = True, \n",
    "            max_length = self.max_len, \n",
    "            return_token_type_ids = False, \n",
    "            return_attention_mask = True, \n",
    "            truncation = True,\n",
    "            pad_to_max_length = True, \n",
    "            return_tensors = 'pt',)\n",
    "\n",
    "        output = {\n",
    "            'input_ids': encoding['input_ids'].flatten(),\n",
    "            'attention_mask': encoding['attention_mask'].flatten(),\n",
    "            'targets': torch.tensor(target, dtype=torch.long)\n",
    "        }\n",
    "        if self.include_raw_text:\n",
    "            output['review_text'] = review\n",
    "            \n",
    "        return output \n",
    "\n",
    "\n",
    "def create_data_loader(df, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE, include_raw_text=False):\n",
    "    ds = GPReviewDataset(\n",
    "        reviews = df.content.to_list(), \n",
    "        targets = df.sentiment.to_list(), \n",
    "        tokenizer = tokenizer, \n",
    "        max_len = max_len,\n",
    "        include_raw_text = include_raw_text)\n",
    "    return DataLoader(ds, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(df, test_size = 0.1, random_state = RANDOM_SEED)\n",
    "df_val, df_test = train_test_split(df_test, test_size = 0.5, random_state = RANDOM_SEED)\n",
    "print(df_train.shape, df_val.shape, df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_loader = create_data_loader(df_train, tokenizer)\n",
    "val_data_loader = create_data_loader(df_val, tokenizer, include_raw_text=True)\n",
    "test_data_loader = create_data_loader(df_test, tokenizer, include_raw_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Testing to see if the data loader works appropriately\n",
    "data = next(iter(train_data_loader))\n",
    "print(data.keys())\n",
    "print(data['input_ids'].shape)\n",
    "print(data['attention_mask'].shape)\n",
    "print(data['targets'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A4sQSt0UkoCu"
   },
   "source": [
    "# Model Utility Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9grQh_PiVx62",
    "outputId": "19d7e504-f682-46e3-8789-139891802596"
   },
   "outputs": [],
   "source": [
    "bert_model = RobertaModel.from_pretrained(PRE_TRAINED_MODEL_NAME)\n",
    "\n",
    "class SentimentClassifier(nn.Module):\n",
    "    def __init__(self, n_classes):\n",
    "        super(SentimentClassifier, self).__init__()\n",
    "        self.bert = RobertaModel.from_pretrained(PRE_TRAINED_MODEL_NAME,return_dict=False)\n",
    "        self.drop = nn.Dropout(p = 0.3)\n",
    "        self.out = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, pooled_output = self.bert(\n",
    "            input_ids = input_ids,\n",
    "            attention_mask= attention_mask\n",
    "        )\n",
    "        output = self.drop(pooled_output)\n",
    "        return self.out(output)\n",
    "  \n",
    "    \n",
    "model = SentimentClassifier(len(class_names))\n",
    "model = model.to(device)\n",
    "\n",
    "# An evaluation run of the model\n",
    "input_ids = data['input_ids'].to(device)\n",
    "attention_mask = data['attention_mask'].to(device)\n",
    "F.softmax(model(input_ids,attention_mask), dim = 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eqO39_ANkrlI"
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "70NX9jLlktnx"
   },
   "source": [
    "## Training Loop Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     9,
     30
    ],
    "id": "VaVkCHUxfRh6"
   },
   "outputs": [],
   "source": [
    "EPOCHS = 10\n",
    "\n",
    "optimizer = optim.AdamW(model.parameters(), lr= 2e-5)\n",
    "total_steps = len(train_data_loader) * EPOCHS\n",
    "scheduler = get_linear_schedule_with_warmup(\n",
    "    optimizer, \n",
    "    num_warmup_steps = 0, \n",
    "    num_training_steps=total_steps)\n",
    "loss_fn = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "\n",
    "def train_epoch(model, data_loader, loss_fn, optimizer, device, scheduler, n_examples):\n",
    "    model=model.train()\n",
    "    losses = []\n",
    "    correct_predictions = 0\n",
    "    \n",
    "    for d in data_loader:\n",
    "        input_ids = d[\"input_ids\"].to(device)\n",
    "        attention_mask = d[\"attention_mask\"].to(device)\n",
    "        targets = d[\"targets\"].to(device)\n",
    "        \n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        _, preds = torch.max(outputs, dim = 1)\n",
    "        loss = loss_fn(outputs, targets)\n",
    "        correct_predictions += torch.sum(preds == targets).cpu()\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "    return correct_predictions/n_examples, np.mean(losses)\n",
    "\n",
    "\n",
    "def eval_model(model, data_loader, loss_fn, device, n_examples):\n",
    "    model = model.eval()\n",
    "    losses = []\n",
    "    correct_predictions = 0\n",
    "  \n",
    "    with torch.no_grad():\n",
    "        for d in data_loader:\n",
    "            input_ids = d[\"input_ids\"].to(device)\n",
    "            attention_mask = d[\"attention_mask\"].to(device)\n",
    "            targets = d[\"targets\"].to(device)\n",
    "\n",
    "            outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "            _,preds = torch.max(outputs, dim = 1)\n",
    "\n",
    "            loss = loss_fn(outputs, targets)\n",
    "            correct_predictions += torch.sum(preds == targets).cpu()\n",
    "            losses.append(loss.item())\n",
    "    return correct_predictions/n_examples, np.mean(losses)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xASNu0l4kzAC"
   },
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0atMu0I8fD0J",
    "outputId": "e9d6d6ba-c3aa-408e-edb5-b47beb63d8a0",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "history = defaultdict(list)\n",
    "best_accuracy = 0\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    print(f'Epoch {epoch + 1}/ {EPOCHS}')\n",
    "    print('-'*15)\n",
    "    train_acc, train_loss = train_epoch(model, train_data_loader, loss_fn, optimizer, device, scheduler, len(df_train))\n",
    "    print(f'Train loss {train_loss} accuracy {train_acc}')\n",
    "\n",
    "    val_acc, val_loss = eval_model(model, val_data_loader, loss_fn, device, len(df_val))\n",
    "    print(f'Val loss {val_loss} accuracy {val_acc}')\n",
    "\n",
    "    history['train_acc'].append(train_acc)\n",
    "    history['train_loss'].append(train_loss)\n",
    "    history['val_acc'].append(val_acc)\n",
    "    history['val_loss'].append(val_loss)\n",
    "  \n",
    "    if val_acc>best_accuracy:\n",
    "        torch.save(model.state_dict(), 'best_model_state.bin')\n",
    "        best_accuracy = val_acc\n",
    "\n",
    "plt.plot(history['train_acc'], label='train accuracy')\n",
    "plt.plot(history['val_acc'], label='validation accuracy')\n",
    "plt.title('Training history')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend()\n",
    "plt.ylim([0,1])\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eiXdRlOfk1-I"
   },
   "source": [
    "## Visualizing Training/Validation Losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hMy7UZbir8gj"
   },
   "outputs": [],
   "source": [
    "plt.plot(history['train_loss'], label='train loss',linewidth=3)\n",
    "plt.plot(history['val_loss'], '--',label='validation loss',linewidth=3)\n",
    "\n",
    "plt.title('Loss history')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend()\n",
    "plt.ylim([0, 1]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Xj1OCvyDNFW8"
   },
   "outputs": [],
   "source": [
    "test_acc, _ = eval_model(model, test_data_loader, loss_fn, device, len(df_test))\n",
    "print(f'Test Accuracy {test_acc.item()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i7NhCQLnk5F3"
   },
   "source": [
    "# Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     1,
     25
    ],
    "id": "cuj1JI24NkJa"
   },
   "outputs": [],
   "source": [
    "def get_predictions(model, data_loader):\n",
    "    model = model.eval()\n",
    "    review_texts = []\n",
    "    predictions = []\n",
    "    prediction_probs = []\n",
    "    real_values = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for d in data_loader:\n",
    "            texts = d[\"review_text\"]\n",
    "            input_ids = d[\"input_ids\"].to(device)\n",
    "            attention_mask = d[\"attention_mask\"].to(device)\n",
    "            targets = d[\"targets\"].to(device)\n",
    "            outputs = model(input_ids = input_ids, attention_mask = attention_mask)\n",
    "            _, preds = torch.max(outputs, dim=1)\n",
    "            probs = F.softmax(outputs, dim =1)\n",
    "            review_texts.extend(texts)\n",
    "            predictions.extend(preds)\n",
    "            prediction_probs.extend(probs)\n",
    "            real_values.extend(targets)\n",
    "    \n",
    "    predictions = torch.stack(predictions).cpu()\n",
    "    prediction_probs = torch.stack(prediction_probs).cpu()\n",
    "    real_values = torch.stack(real_values).cpu()\n",
    "    return review_texts, predictions, prediction_probs, real_values\n",
    "\n",
    "def show_confusion_matrix(confusion_matrix):\n",
    "    hmap = sns.heatmap(confusion_matrix, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "    hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation = 0, ha='right')\n",
    "    hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation = 30, ha='right')\n",
    "    plt.ylabel('True Sentiment')\n",
    "    plt.xlabel('Predicted Sentiment')\n",
    "\n",
    "y_review_texts, y_pred, y_pred_probs, y_test = get_predictions(model, test_data_loader)\n",
    "print(classification_report(y_test, y_pred, target_names=class_names))\n",
    "\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "df_cm = pd.DataFrame(cm, index=class_names, columns = class_names)\n",
    "show_confusion_matrix(df_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "r4dropvXT-T3"
   },
   "outputs": [],
   "source": [
    "idx = 2\n",
    "review_text = y_review_texts[idx]\n",
    "true_sentiment = y_test[idx]\n",
    "pred_df = pd.DataFrame({'class_names':class_names, 'values':y_pred_probs[idx]})\n",
    "print(\"\\n\".join(wrap(review_text)))\n",
    "print()\n",
    "print(f'True Sentiment: {class_names[true_sentiment]}')\n",
    "\n",
    "sns.barplot(x='values', y='class_names', data=pred_df, orient='h')\n",
    "plt.ylabel('sentiment')\n",
    "plt.xlabel('probability')\n",
    "plt.xlim([0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vrvFFMASWigO"
   },
   "outputs": [],
   "source": [
    "review_text = \"I love completing my todos! Best app ever!!!\"\n",
    "encoded_review = tokenizer.encode_plus(review_text, max_length=MAX_LEN, add_special_tokens=True, return_token_type_ids=False, pad_to_max_length=True, return_attention_mask=True,\n",
    "                                       truncation=True, return_tensors='pt')\n",
    "input_ids = encoded_review['input_ids'].to(device)\n",
    "attention_mask=encoded_review['attention_mask'].to(device)\n",
    "output = model(input_ids, attention_mask)\n",
    "_,prediction = torch.max(output, dim=1)\n",
    "\n",
    "print(f'Review text: {review_text}')\n",
    "print(f'Sentiment  : {class_names[prediction]}')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ma6GjPBnA8Ta"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Ch3-BERT_SentimentClassification-roberta-TransformersBook.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "DeepLearning",
   "language": "python",
   "name": "deeplearn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
